/*
 ******************
 * author: 马丰彦
 * date: 2019/1/14
 * function: 详情页爬取
 ******************
 */

const readline = require('readline');
var url = require('./model/moreLinks'); // 引入电影url模型
var firmRank = require('./model/filmRanking'); // 引入电影详情模型
var Crawler = require('crawler'); // node-crawler爬虫框架
var repeatDetail = require('./model/repeatDetail'); // 引入查重模型
var mongoose = require('mongoose');
mongoose.connect('mongodb://127.0.0.1"27017/film', {
  useNewUrlParser: true
})
const db = mongoose.connection
db.once('open', function () {
  console.log('连接数据库成功\n');
  // 电影详情页爬取
  repeatDetail.countDocuments({}, function (error, docs) {
    if (error) {
      console.log(error);
    } else {
      var num = docs;
      var indexs = num
      const rl = readline.createInterface({
        input: process.stdin,
        output: process.stdout,
        prompt: '请输入代理ip(直接按enter为不填)> '
      });
      rl.prompt();
      rl.on('line', (line) => {
        var http = null;
        if (!!line) {
          http = 'http://' + line;
        }
        var firmId = 0;
        var frimDetail = new Crawler({
          rateLimit: 2000,
          jQuery: true,
          proxy: http,
          callback: async function (error, res, done) {
            if (error) {
              console.log(error);
            } else {
              var $ = res.$;
              var forbidden = $('center').text();
              if (forbidden.indexOf('403 Forbiddennginx') >= 0 || res.body.indexOf('检测到有异常请求从你的') >= 0) {
                console.log('IP被封掉了,咋整啊？？才爬到第' + indexs + '个！QAQ\n');
                stopDetail();
                rl.prompt();
              } else {
                var $ = res.$;
                var indexNum = res.options.uri.indexOf('/subject/') + 9;
                var numbersIndex = res.options.uri.substr(indexNum);
                firmId = numbersIndex.replace('/', ''); // 获取电影id
                // 得到标题
                var text = $('#content').find('h1').children('span').text();
                var firmTitle = text.replace(/\(.+\)/, '') // 正则去掉标题括号年份
                // 得到图片
                var firmImg = $('#mainpic').find('img').attr('src');
                var info = $('#info');
                // 导演
                var firmDirectors = info.children('span').first().children('span').last().text();
                // 编剧
                var firmScreenWriter = info.children('span').eq(1).children('span').last().text();
                // 主演
                var firmActior = info.children('span').eq(2).children('span').last().text();
                // 类型
                var span = [];
                info.children('span').each(function (item) {
                  span.push($(this).text())
                });
                var city = span.indexOf('制片国家/地区:');
                var types = []
                for (let i = 4; i < city; i++) {
                  types.push(span[i]);
                }
                var firmType = types.join(',').replace(',官方网站:', ''); // 类型
                var word = info.text();
                var indexWord = word.indexOf('制片国家/地区:');
                var indexLangu = word.indexOf('语言:');
                // 国家
                var firmCountry = '';
                var show = word.indexOf('上映日期:');
                // 语言
                var firmLanguage = '';
                if (indexWord >= 0 && indexLangu >= 0) {
                  firmCountry = word.substring(indexWord + 8, indexLangu).replace('\n', '').trim();
                }
                if (show >= 0 && indexLangu >= 0) {
                  firmLanguage = word.substring(indexLangu + 3, show).replace('\n', '').trim();
                }

                var time = word.indexOf('片长:');
                var anotherName = word.indexOf('又名:');
                // 片长
                var firmTime = '';
                if (time >= 0 && anotherName >= 0) {
                  firmTime = word.substring(time + 3, anotherName).replace('\n', '').trim();
                }
                // 评分
                var point = $('#interest_sectl');
                var firmPoint = point.find('strong').text();
                // 人数
                var firmTalkNum = point.find('a.rating_people').text();
                // 电影简介
                var firmIntro = $('#link-report').find('span').first().text().replace('\n', '').trim();
                var firmResult = {
                  firmTitle: firmTitle,
                  firmImg: firmImg,
                  firmDirectors: firmDirectors,
                  firmScreenWriter: firmScreenWriter,
                  firmActior: firmActior,
                  firmType: firmType,
                  firmCountry: firmCountry,
                  firmLanguage: firmLanguage,
                  firmTime: firmTime,
                  firmPoint: firmPoint,
                  firmTalkNum: firmTalkNum,
                  firmIntro: firmIntro,
                  firmId: firmId
                }
                await firmRank.updateOne({
                  firmId: firmId
                }, firmResult, {
                  upsert: true
                }, function (error, docs) {
                  if (error) {
                    console.log(error);
                  } else {
                    console.log('序号：' + indexs + '! 成功! 电影名:' + firmTitle + '\n');
                    indexs++
                  }
                })
                await repeatDetail.insertMany({
                  url: res.options.uri
                }, function (error, docs) {
                  if (error) {
                    console.log(error);
                  }
                })
              }
            }
            done();
          }
        })

        function searchUrl() {
          url.findOne({}, function (error, docs) { // 从url的collection中调取url
            if (docs === null) {
              console.log('完事了');
              stopDetail(); // 停止爬取
            } else {
              repeatDetail.findOne({
                url: docs.url
              }, function (error, item) { // 查重减少爬取
                if (error) {
                  console.log(error);
                } else {
                  if (item === null) {
                    frimDetail.queue(docs.url);
                  } else {
                    console.log('重复url\n');
                  }
                }
              })
            }
          }).skip(num).limit(1)
          num++;
        }
        var myDetail = setInterval(searchUrl, 1000);

        function stopDetail() {
          clearInterval(myDetail)
        }
      }).on('close', () => {
        console.log('你已退出，已爬取' + indexs + '部电影');
        process.exit(0);
      });
    }
  })
})
db.on('error', function (error) {
  console.log('连接失败' + error);
});
db.on('disconnected', function () {
  console.log('连接断开');
})